{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "from typing import *\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from functools import partial\n",
    "from overrides import overrides\n",
    "\n",
    "from allennlp.data import Instance\n",
    "from allennlp.data.token_indexers import TokenIndexer\n",
    "from allennlp.data.tokenizers import Token\n",
    "from allennlp.nn import util as nn_util"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Config(dict):\n",
    "    def __init__(self, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        for k, v in kwargs.items():\n",
    "            setattr(self, k, v)\n",
    "    \n",
    "    def set(self, key, val):\n",
    "        self[key] = val\n",
    "        setattr(self, key, val)\n",
    "        \n",
    "config = Config(\n",
    "    testing=True,\n",
    "    seed=1,\n",
    "    batch_size=64,\n",
    "    lr=3e-4,\n",
    "    epochs=2,\n",
    "    hidden_sz=64,\n",
    "    max_seq_len=100, # necessary to limit memory usage\n",
    "    max_vocab_size=100000,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from allennlp.common.checks import ConfigurationError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "USE_GPU = torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_ROOT = Path(\"../data\") / \"jigsaw\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set random seed manually to replicate results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x10deec650>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manual_seed(config.seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from allennlp.data.vocabulary import Vocabulary\n",
    "from allennlp.data.dataset_readers import DatasetReader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_cols = [\"toxic\", \"severe_toxic\", \"obscene\",\n",
    "              \"threat\", \"insult\", \"identity_hate\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from allennlp.data.fields import TextField, MetadataField, ArrayField\n",
    "\n",
    "class JigsawDatasetReader(DatasetReader):\n",
    "    def __init__(self, tokenizer: Callable[[str], List[str]]=lambda x: x.split(),\n",
    "                 token_indexers: Dict[str, TokenIndexer] = None,\n",
    "                 max_seq_len: Optional[int]=config.max_seq_len) -> None:\n",
    "        super().__init__(lazy=False)\n",
    "        self.tokenizer = tokenizer\n",
    "        self.token_indexers = token_indexers or {\"tokens\": SingleIdTokenIndexer()}\n",
    "        self.max_seq_len = max_seq_len\n",
    "\n",
    "    @overrides\n",
    "    def text_to_instance(self, tokens: List[Token], id: str=None,\n",
    "                         labels: np.ndarray=None) -> Instance:\n",
    "        sentence_field = TextField(tokens, self.token_indexers)\n",
    "        fields = {\"tokens\": sentence_field}\n",
    "        \n",
    "        id_field = MetadataField(id)\n",
    "        fields[\"id\"] = id_field\n",
    "        \n",
    "        if labels is None:\n",
    "            labels = np.zeros(len(label_cols))\n",
    "        label_field = ArrayField(array=labels)\n",
    "        fields[\"label\"] = label_field\n",
    "\n",
    "        return Instance(fields)\n",
    "    \n",
    "    @overrides\n",
    "    def _read(self, file_path: str) -> Iterator[Instance]:\n",
    "        df = pd.read_csv(file_path)\n",
    "        if config.testing: df = df.head(1000)\n",
    "        for i, row in df.iterrows():\n",
    "            yield self.text_to_instance(\n",
    "                [Token(x) for x in self.tokenizer(row[\"comment_text\"])],\n",
    "                row[\"id\"], row[label_cols].values,\n",
    "            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare token handlers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will use the spacy tokenizer here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from allennlp.data.tokenizers.word_splitter import SpacyWordSplitter\n",
    "from allennlp.data.token_indexers import SingleIdTokenIndexer\n",
    "\n",
    "# the token indexer is responsible for mapping tokens to integers\n",
    "token_indexer = SingleIdTokenIndexer()\n",
    "\n",
    "def tokenizer(x: str):\n",
    "    return [w.text for w in\n",
    "            SpacyWordSplitter(language='en_core_web_sm', \n",
    "                              pos_tags=False).split_words(x)[:config.max_seq_len]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "reader = JigsawDatasetReader(\n",
    "    tokenizer=tokenizer,\n",
    "    token_indexers={\"tokens\": token_indexer}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "267it [00:01, 139.37it/s]\n",
      "251it [00:00, 266.95it/s]\n"
     ]
    }
   ],
   "source": [
    "train_ds, test_ds = (reader.read(DATA_ROOT / fname) for fname in [\"train.csv\", \"test_proced.csv\"])\n",
    "val_ds = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "267"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<allennlp.data.instance.Instance at 0x1a281c0208>,\n",
       " <allennlp.data.instance.Instance at 0x1a281a3dd8>,\n",
       " <allennlp.data.instance.Instance at 0x1a2817aa58>,\n",
       " <allennlp.data.instance.Instance at 0x1a28120c18>,\n",
       " <allennlp.data.instance.Instance at 0x1a28117048>,\n",
       " <allennlp.data.instance.Instance at 0x1a2810c6d8>,\n",
       " <allennlp.data.instance.Instance at 0x1a28105198>,\n",
       " <allennlp.data.instance.Instance at 0x1a28120588>,\n",
       " <allennlp.data.instance.Instance at 0x1a25eeb128>,\n",
       " <allennlp.data.instance.Instance at 0x1a25f0c550>]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_ds[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'tokens': [Explanation,\n",
       "  Why,\n",
       "  the,\n",
       "  edits,\n",
       "  made,\n",
       "  under,\n",
       "  my,\n",
       "  username,\n",
       "  Hardcore,\n",
       "  Metallica,\n",
       "  Fan,\n",
       "  were,\n",
       "  reverted,\n",
       "  ?,\n",
       "  They,\n",
       "  were,\n",
       "  n't,\n",
       "  vandalisms,\n",
       "  ,,\n",
       "  just,\n",
       "  closure,\n",
       "  on,\n",
       "  some,\n",
       "  GAs,\n",
       "  after,\n",
       "  I,\n",
       "  voted,\n",
       "  at,\n",
       "  New,\n",
       "  York,\n",
       "  Dolls,\n",
       "  FAC,\n",
       "  .,\n",
       "  And,\n",
       "  please,\n",
       "  do,\n",
       "  n't,\n",
       "  remove,\n",
       "  the,\n",
       "  template,\n",
       "  from,\n",
       "  the,\n",
       "  talk,\n",
       "  page,\n",
       "  since,\n",
       "  I,\n",
       "  'm,\n",
       "  retired,\n",
       "  now.89.205.38.27],\n",
       " '_token_indexers': {'tokens': <allennlp.data.token_indexers.single_id_token_indexer.SingleIdTokenIndexer at 0x1a24c8c080>},\n",
       " '_indexed_tokens': None,\n",
       " '_indexer_name_to_indexed_token': None}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vars(train_ds[0].fields[\"tokens\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "02/07/2019 17:53:35 - INFO - allennlp.data.vocabulary -   Fitting token dictionary from dataset.\n",
      "100%|██████████| 267/267 [00:00<00:00, 14279.62it/s]\n"
     ]
    }
   ],
   "source": [
    "vocab = Vocabulary.from_instances(train_ds, max_vocab_size=config.max_vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare iterator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The iterator is responsible for batching the data and preparing it for input into the model. We'll use the BucketIterator that batches text sequences of smilar lengths together."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from allennlp.data.iterators import BucketIterator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "iterator = BucketIterator(batch_size=config.batch_size, \n",
    "                          sorting_keys=[(\"tokens\", \"num_tokens\")],\n",
    "                         )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We need to tell the iterator how to numericalize the text data. We do this by passing the vocabulary to the iterator. This step is easy to forget so be careful! "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "iterator.index_with(vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch = next(iter(iterator(train_ds)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'tokens': {'tokens': tensor([[2952,  166,   34,  ...,    0,    0,    0],\n",
       "          [  42,  613,  986,  ...,    0,    0,    0],\n",
       "          [ 385,   94,   95,  ...,    0,    0,    0],\n",
       "          ...,\n",
       "          [   5,  498,   45,  ...,    4,  749,    3],\n",
       "          [   5,  672,  107,  ...,    6,  374,    8],\n",
       "          [ 262,   46,    4,  ..., 1210,    0,    0]])},\n",
       " 'id': ['008344a80c43b8c9',\n",
       "  '005306a4c109dab3',\n",
       "  '008f1b06428778fe',\n",
       "  '002264ea4d5f2887',\n",
       "  '0038f191ffc93d75',\n",
       "  '006d11791d76b9f3',\n",
       "  '001d8e7be417776a',\n",
       "  '004176f28a17bf45',\n",
       "  '00a981826dc7c952',\n",
       "  '005e6b1369cbe377',\n",
       "  '00a317acddff8a62',\n",
       "  '006a4cdda4960588',\n",
       "  '002746baedcdff10',\n",
       "  '007e1e47cd0e2fec',\n",
       "  '00151a9f93c6b059',\n",
       "  '001cadfd324f8087',\n",
       "  '000cfee90f50d471',\n",
       "  '0016e01b742b8da3',\n",
       "  '0006f16e4e9f292e',\n",
       "  '001ee16c46a99262',\n",
       "  '003b9f448ee4a29d',\n",
       "  '0060062dd4db5195',\n",
       "  '0048de0c9422f64f',\n",
       "  '001d874a4d3e8813',\n",
       "  '004b103182fb1eab',\n",
       "  '00822d0d01752c7e',\n",
       "  '00548f16a392d8ed',\n",
       "  '009565ee1bc64e68',\n",
       "  '001810bf8c45bf5f',\n",
       "  '0091aec11b57d12e',\n",
       "  '00a66540b2a66bb4',\n",
       "  '0063a8786a7034fc',\n",
       "  '00957fadc476d7d9',\n",
       "  '0038d1dc2ad29469',\n",
       "  '0033b9d5ccd499fb',\n",
       "  '001363e1dbe91225',\n",
       "  '00584d887401f47b',\n",
       "  '006774d59329b7bd',\n",
       "  '00a6754c92b4af4f',\n",
       "  '002c9cccf2f1d05b',\n",
       "  '00587c559177dcf2',\n",
       "  '000c0dfd995809fa',\n",
       "  '007db1f1477ea977',\n",
       "  '009820fe28cd24dc',\n",
       "  '005ed4dfecd86188',\n",
       "  '0021fe88bc4da3e6',\n",
       "  '00510c3d06745849',\n",
       "  '004b073d5b456b15',\n",
       "  '00218d74784ce50b',\n",
       "  '00328eadb85b3010',\n",
       "  '008a1e9c45de8138',\n",
       "  '007f127033d66db5',\n",
       "  '003a19c04c079bf7',\n",
       "  '0063f66706c20dfa',\n",
       "  '004b975fabbbffa9',\n",
       "  '005cec874506e9d9',\n",
       "  '004d07d94cb92e35',\n",
       "  '0034065c7b12a7a2',\n",
       "  '00078f8ce7eb276d',\n",
       "  '005f59485fcddeb0',\n",
       "  '00480b6e1f19601b',\n",
       "  '00148d055a169b93',\n",
       "  '0005300084f90edc',\n",
       "  '00037261f536c51d'],\n",
       " 'label': tensor([[0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [1., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 1., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [1., 0., 1., 0., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [1., 0., 1., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [1., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0.]])}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[2952,  166,   34,  ...,    0,    0,    0],\n",
       "        [  42,  613,  986,  ...,    0,    0,    0],\n",
       "        [ 385,   94,   95,  ...,    0,    0,    0],\n",
       "        ...,\n",
       "        [   5,  498,   45,  ...,    4,  749,    3],\n",
       "        [   5,  672,  107,  ...,    6,  374,    8],\n",
       "        [ 262,   46,    4,  ..., 1210,    0,    0]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch[\"tokens\"][\"tokens\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([64, 100])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch[\"tokens\"][\"tokens\"].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from allennlp.modules.seq2vec_encoders import Seq2VecEncoder, PytorchSeq2VecWrapper\n",
    "from allennlp.nn.util import get_text_field_mask\n",
    "from allennlp.models import Model\n",
    "from allennlp.modules.text_field_embedders import TextFieldEmbedder\n",
    "\n",
    "class BaselineModel(Model):\n",
    "    def __init__(self, word_embeddings: TextFieldEmbedder,\n",
    "                 encoder: Seq2VecEncoder,\n",
    "                 out_sz: int=len(label_cols)):\n",
    "        super().__init__(vocab)\n",
    "        self.word_embeddings = word_embeddings\n",
    "        self.encoder = encoder\n",
    "        self.projection = nn.Linear(self.encoder.get_output_dim(), out_sz)\n",
    "        self.loss = nn.BCEWithLogitsLoss()\n",
    "        \n",
    "    def forward(self, tokens: Dict[str, torch.Tensor],\n",
    "                id: Any, label: torch.Tensor) -> torch.Tensor:\n",
    "        mask = get_text_field_mask(tokens)\n",
    "        embeddings = self.word_embeddings(tokens)\n",
    "        state = self.encoder(embeddings, mask)\n",
    "        class_logits = self.projection(state)\n",
    "        \n",
    "        output = {\"class_logits\": class_logits}\n",
    "        output[\"loss\"] = self.loss(class_logits, label)\n",
    "\n",
    "        return output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from allennlp.modules.token_embedders import Embedding\n",
    "from allennlp.modules.text_field_embedders import BasicTextFieldEmbedder\n",
    "\n",
    "token_embedding = Embedding(num_embeddings=config.max_vocab_size + 2,\n",
    "                            embedding_dim=300, padding_index=0)\n",
    "# the embedder maps the input tokens to the appropriate embedding matrix\n",
    "word_embeddings: TextFieldEmbedder = BasicTextFieldEmbedder({\"tokens\": token_embedding})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from allennlp.modules.seq2vec_encoders import PytorchSeq2VecWrapper\n",
    "encoder: Seq2VecEncoder = PytorchSeq2VecWrapper(nn.LSTM(word_embeddings.get_output_dim(),\n",
    "                                                        config.hidden_sz, bidirectional=True, batch_first=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notice how simple and modular the code for initializing the model is. All the complexity is delegated to each component."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = BaselineModel(\n",
    "    word_embeddings, \n",
    "    encoder, \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "if USE_GPU: model.cuda()\n",
    "else: model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Basic sanity checks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch = nn_util.move_to_device(batch, 0 if USE_GPU else -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = batch[\"tokens\"]\n",
    "labels = batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'tokens': tensor([[2952,  166,   34,  ...,    0,    0,    0],\n",
       "         [  42,  613,  986,  ...,    0,    0,    0],\n",
       "         [ 385,   94,   95,  ...,    0,    0,    0],\n",
       "         ...,\n",
       "         [   5,  498,   45,  ...,    4,  749,    3],\n",
       "         [   5,  672,  107,  ...,    6,  374,    8],\n",
       "         [ 262,   46,    4,  ..., 1210,    0,    0]])}"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1, 1, 1,  ..., 0, 0, 0],\n",
       "        [1, 1, 1,  ..., 0, 0, 0],\n",
       "        [1, 1, 1,  ..., 0, 0, 0],\n",
       "        ...,\n",
       "        [1, 1, 1,  ..., 1, 1, 1],\n",
       "        [1, 1, 1,  ..., 1, 1, 1],\n",
       "        [1, 1, 1,  ..., 1, 0, 0]])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mask = get_text_field_mask(tokens)\n",
    "mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0383,  0.0800, -0.0317,  0.0283,  0.0172, -0.0236],\n",
       "        [-0.0398,  0.0806, -0.0331,  0.0289,  0.0170, -0.0243],\n",
       "        [-0.0387,  0.0807, -0.0297,  0.0295,  0.0170, -0.0230],\n",
       "        [-0.0386,  0.0815, -0.0327,  0.0297,  0.0152, -0.0263],\n",
       "        [-0.0367,  0.0808, -0.0306,  0.0292,  0.0149, -0.0246],\n",
       "        [-0.0390,  0.0815, -0.0305,  0.0276,  0.0160, -0.0249],\n",
       "        [-0.0393,  0.0814, -0.0302,  0.0274,  0.0173, -0.0252],\n",
       "        [-0.0396,  0.0805, -0.0303,  0.0269,  0.0167, -0.0240],\n",
       "        [-0.0379,  0.0795, -0.0300,  0.0281,  0.0144, -0.0260],\n",
       "        [-0.0385,  0.0804, -0.0310,  0.0263,  0.0171, -0.0263],\n",
       "        [-0.0387,  0.0807, -0.0317,  0.0268,  0.0157, -0.0245],\n",
       "        [-0.0381,  0.0792, -0.0317,  0.0292,  0.0151, -0.0237],\n",
       "        [-0.0371,  0.0806, -0.0309,  0.0270,  0.0156, -0.0257],\n",
       "        [-0.0389,  0.0818, -0.0308,  0.0271,  0.0154, -0.0245],\n",
       "        [-0.0372,  0.0807, -0.0321,  0.0259,  0.0154, -0.0235],\n",
       "        [-0.0363,  0.0805, -0.0310,  0.0265,  0.0162, -0.0247],\n",
       "        [-0.0386,  0.0804, -0.0321,  0.0266,  0.0161, -0.0239],\n",
       "        [-0.0396,  0.0812, -0.0313,  0.0287,  0.0171, -0.0251],\n",
       "        [-0.0396,  0.0810, -0.0299,  0.0276,  0.0167, -0.0256],\n",
       "        [-0.0377,  0.0792, -0.0330,  0.0273,  0.0160, -0.0254],\n",
       "        [-0.0391,  0.0809, -0.0309,  0.0271,  0.0174, -0.0245],\n",
       "        [-0.0386,  0.0802, -0.0309,  0.0266,  0.0165, -0.0252],\n",
       "        [-0.0379,  0.0798, -0.0332,  0.0267,  0.0166, -0.0253],\n",
       "        [-0.0381,  0.0792, -0.0318,  0.0271,  0.0152, -0.0248],\n",
       "        [-0.0403,  0.0802, -0.0311,  0.0291,  0.0169, -0.0251],\n",
       "        [-0.0380,  0.0791, -0.0311,  0.0261,  0.0157, -0.0235],\n",
       "        [-0.0378,  0.0803, -0.0304,  0.0289,  0.0160, -0.0266],\n",
       "        [-0.0379,  0.0801, -0.0300,  0.0279,  0.0164, -0.0259],\n",
       "        [-0.0389,  0.0804, -0.0308,  0.0298,  0.0162, -0.0268],\n",
       "        [-0.0385,  0.0803, -0.0323,  0.0275,  0.0163, -0.0235],\n",
       "        [-0.0379,  0.0802, -0.0313,  0.0283,  0.0164, -0.0240],\n",
       "        [-0.0374,  0.0800, -0.0297,  0.0260,  0.0171, -0.0252],\n",
       "        [-0.0382,  0.0801, -0.0310,  0.0268,  0.0163, -0.0242],\n",
       "        [-0.0396,  0.0817, -0.0315,  0.0293,  0.0181, -0.0266],\n",
       "        [-0.0375,  0.0808, -0.0321,  0.0270,  0.0166, -0.0245],\n",
       "        [-0.0388,  0.0809, -0.0304,  0.0279,  0.0169, -0.0251],\n",
       "        [-0.0396,  0.0798, -0.0312,  0.0293,  0.0159, -0.0251],\n",
       "        [-0.0396,  0.0814, -0.0302,  0.0277,  0.0183, -0.0252],\n",
       "        [-0.0384,  0.0818, -0.0313,  0.0275,  0.0161, -0.0249],\n",
       "        [-0.0371,  0.0795, -0.0327,  0.0270,  0.0161, -0.0241],\n",
       "        [-0.0378,  0.0795, -0.0299,  0.0272,  0.0172, -0.0245],\n",
       "        [-0.0366,  0.0810, -0.0316,  0.0265,  0.0165, -0.0238],\n",
       "        [-0.0388,  0.0801, -0.0317,  0.0291,  0.0172, -0.0256],\n",
       "        [-0.0397,  0.0819, -0.0319,  0.0282,  0.0177, -0.0249],\n",
       "        [-0.0375,  0.0815, -0.0310,  0.0260,  0.0146, -0.0248],\n",
       "        [-0.0382,  0.0797, -0.0317,  0.0278,  0.0168, -0.0264],\n",
       "        [-0.0400,  0.0806, -0.0304,  0.0284,  0.0178, -0.0255],\n",
       "        [-0.0391,  0.0803, -0.0306,  0.0270,  0.0168, -0.0246],\n",
       "        [-0.0382,  0.0800, -0.0319,  0.0274,  0.0153, -0.0240],\n",
       "        [-0.0375,  0.0803, -0.0302,  0.0261,  0.0166, -0.0242],\n",
       "        [-0.0371,  0.0797, -0.0311,  0.0261,  0.0166, -0.0240],\n",
       "        [-0.0384,  0.0801, -0.0314,  0.0281,  0.0175, -0.0241],\n",
       "        [-0.0373,  0.0814, -0.0303,  0.0277,  0.0147, -0.0251],\n",
       "        [-0.0387,  0.0794, -0.0314,  0.0267,  0.0164, -0.0232],\n",
       "        [-0.0371,  0.0799, -0.0300,  0.0268,  0.0144, -0.0264],\n",
       "        [-0.0387,  0.0787, -0.0305,  0.0272,  0.0173, -0.0241],\n",
       "        [-0.0401,  0.0804, -0.0321,  0.0297,  0.0171, -0.0254],\n",
       "        [-0.0393,  0.0790, -0.0311,  0.0288,  0.0180, -0.0260],\n",
       "        [-0.0381,  0.0802, -0.0305,  0.0272,  0.0154, -0.0267],\n",
       "        [-0.0380,  0.0815, -0.0317,  0.0272,  0.0170, -0.0245],\n",
       "        [-0.0382,  0.0799, -0.0315,  0.0286,  0.0156, -0.0259],\n",
       "        [-0.0378,  0.0807, -0.0309,  0.0268,  0.0151, -0.0257],\n",
       "        [-0.0385,  0.0805, -0.0304,  0.0270,  0.0154, -0.0259],\n",
       "        [-0.0382,  0.0797, -0.0315,  0.0275,  0.0136, -0.0265]],\n",
       "       grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeddings = model.word_embeddings(tokens)\n",
    "state = model.encoder(embeddings, mask)\n",
    "class_logits = model.projection(state)\n",
    "class_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'class_logits': tensor([[-0.0383,  0.0800, -0.0317,  0.0283,  0.0172, -0.0236],\n",
       "         [-0.0398,  0.0806, -0.0331,  0.0289,  0.0170, -0.0243],\n",
       "         [-0.0387,  0.0807, -0.0297,  0.0295,  0.0170, -0.0230],\n",
       "         [-0.0386,  0.0815, -0.0327,  0.0297,  0.0152, -0.0263],\n",
       "         [-0.0367,  0.0808, -0.0306,  0.0292,  0.0149, -0.0246],\n",
       "         [-0.0390,  0.0815, -0.0305,  0.0276,  0.0160, -0.0249],\n",
       "         [-0.0393,  0.0814, -0.0302,  0.0274,  0.0173, -0.0252],\n",
       "         [-0.0396,  0.0805, -0.0303,  0.0269,  0.0167, -0.0240],\n",
       "         [-0.0379,  0.0795, -0.0300,  0.0281,  0.0144, -0.0260],\n",
       "         [-0.0385,  0.0804, -0.0310,  0.0263,  0.0171, -0.0263],\n",
       "         [-0.0387,  0.0807, -0.0317,  0.0268,  0.0157, -0.0245],\n",
       "         [-0.0381,  0.0792, -0.0317,  0.0292,  0.0151, -0.0237],\n",
       "         [-0.0371,  0.0806, -0.0309,  0.0270,  0.0156, -0.0257],\n",
       "         [-0.0389,  0.0818, -0.0308,  0.0271,  0.0154, -0.0245],\n",
       "         [-0.0372,  0.0807, -0.0321,  0.0259,  0.0154, -0.0235],\n",
       "         [-0.0363,  0.0805, -0.0310,  0.0265,  0.0162, -0.0247],\n",
       "         [-0.0386,  0.0804, -0.0321,  0.0266,  0.0161, -0.0239],\n",
       "         [-0.0396,  0.0812, -0.0313,  0.0287,  0.0171, -0.0251],\n",
       "         [-0.0396,  0.0810, -0.0299,  0.0276,  0.0167, -0.0256],\n",
       "         [-0.0377,  0.0792, -0.0330,  0.0273,  0.0160, -0.0254],\n",
       "         [-0.0391,  0.0809, -0.0309,  0.0271,  0.0174, -0.0245],\n",
       "         [-0.0386,  0.0802, -0.0309,  0.0266,  0.0165, -0.0252],\n",
       "         [-0.0379,  0.0798, -0.0332,  0.0267,  0.0166, -0.0253],\n",
       "         [-0.0381,  0.0792, -0.0318,  0.0271,  0.0152, -0.0248],\n",
       "         [-0.0403,  0.0802, -0.0311,  0.0291,  0.0169, -0.0251],\n",
       "         [-0.0380,  0.0791, -0.0311,  0.0261,  0.0157, -0.0235],\n",
       "         [-0.0378,  0.0803, -0.0304,  0.0289,  0.0160, -0.0266],\n",
       "         [-0.0379,  0.0801, -0.0300,  0.0279,  0.0164, -0.0259],\n",
       "         [-0.0389,  0.0804, -0.0308,  0.0298,  0.0162, -0.0268],\n",
       "         [-0.0385,  0.0803, -0.0323,  0.0275,  0.0163, -0.0235],\n",
       "         [-0.0379,  0.0802, -0.0313,  0.0283,  0.0164, -0.0240],\n",
       "         [-0.0374,  0.0800, -0.0297,  0.0260,  0.0171, -0.0252],\n",
       "         [-0.0382,  0.0801, -0.0310,  0.0268,  0.0163, -0.0242],\n",
       "         [-0.0396,  0.0817, -0.0315,  0.0293,  0.0181, -0.0266],\n",
       "         [-0.0375,  0.0808, -0.0321,  0.0270,  0.0166, -0.0245],\n",
       "         [-0.0388,  0.0809, -0.0304,  0.0279,  0.0169, -0.0251],\n",
       "         [-0.0396,  0.0798, -0.0312,  0.0293,  0.0159, -0.0251],\n",
       "         [-0.0396,  0.0814, -0.0302,  0.0277,  0.0183, -0.0252],\n",
       "         [-0.0384,  0.0818, -0.0313,  0.0275,  0.0161, -0.0249],\n",
       "         [-0.0371,  0.0795, -0.0327,  0.0270,  0.0161, -0.0241],\n",
       "         [-0.0378,  0.0795, -0.0299,  0.0272,  0.0172, -0.0245],\n",
       "         [-0.0366,  0.0810, -0.0316,  0.0265,  0.0165, -0.0238],\n",
       "         [-0.0388,  0.0801, -0.0317,  0.0291,  0.0172, -0.0256],\n",
       "         [-0.0397,  0.0819, -0.0319,  0.0282,  0.0177, -0.0249],\n",
       "         [-0.0375,  0.0815, -0.0310,  0.0260,  0.0146, -0.0248],\n",
       "         [-0.0382,  0.0797, -0.0317,  0.0278,  0.0168, -0.0264],\n",
       "         [-0.0400,  0.0806, -0.0304,  0.0284,  0.0178, -0.0255],\n",
       "         [-0.0391,  0.0803, -0.0306,  0.0270,  0.0168, -0.0246],\n",
       "         [-0.0382,  0.0800, -0.0319,  0.0274,  0.0153, -0.0240],\n",
       "         [-0.0375,  0.0803, -0.0302,  0.0261,  0.0166, -0.0242],\n",
       "         [-0.0371,  0.0797, -0.0311,  0.0261,  0.0166, -0.0240],\n",
       "         [-0.0384,  0.0801, -0.0314,  0.0281,  0.0175, -0.0241],\n",
       "         [-0.0373,  0.0814, -0.0303,  0.0277,  0.0147, -0.0251],\n",
       "         [-0.0387,  0.0794, -0.0314,  0.0267,  0.0164, -0.0232],\n",
       "         [-0.0371,  0.0799, -0.0300,  0.0268,  0.0144, -0.0264],\n",
       "         [-0.0387,  0.0787, -0.0305,  0.0272,  0.0173, -0.0241],\n",
       "         [-0.0401,  0.0804, -0.0321,  0.0297,  0.0171, -0.0254],\n",
       "         [-0.0393,  0.0790, -0.0311,  0.0288,  0.0180, -0.0260],\n",
       "         [-0.0381,  0.0802, -0.0305,  0.0272,  0.0154, -0.0267],\n",
       "         [-0.0380,  0.0815, -0.0317,  0.0272,  0.0170, -0.0245],\n",
       "         [-0.0382,  0.0799, -0.0315,  0.0286,  0.0156, -0.0259],\n",
       "         [-0.0378,  0.0807, -0.0309,  0.0268,  0.0151, -0.0257],\n",
       "         [-0.0385,  0.0805, -0.0304,  0.0270,  0.0154, -0.0259],\n",
       "         [-0.0382,  0.0797, -0.0315,  0.0275,  0.0136, -0.0265]],\n",
       "        grad_fn=<AddmmBackward>),\n",
       " 'loss': tensor(0.6964, grad_fn=<BinaryCrossEntropyWithLogitsBackward>)}"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model(**batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss = model(**batch)[\"loss\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.6964, grad_fn=<BinaryCrossEntropyWithLogitsBackward>)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.Adam(model.parameters(), lr=config.lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "from allennlp.training.trainer import Trainer\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    optimizer=optimizer,\n",
    "    iterator=iterator,\n",
    "    train_dataset=train_ds,\n",
    "    cuda_device=0 if USE_GPU else -1,\n",
    "    num_epochs=config.epochs,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "02/07/2019 17:53:40 - INFO - allennlp.training.trainer -   Beginning training.\n",
      "02/07/2019 17:53:40 - INFO - allennlp.training.trainer -   Epoch 0/1\n",
      "02/07/2019 17:53:40 - INFO - allennlp.training.trainer -   Peak CPU memory usage MB: 724.332544\n",
      "02/07/2019 17:53:40 - INFO - allennlp.training.trainer -   Training\n",
      "loss: 0.6926 ||: 100%|██████████| 5/5 [00:04<00:00,  1.22it/s]\n",
      "02/07/2019 17:53:45 - INFO - allennlp.training.trainer -                     Training |  Validation\n",
      "02/07/2019 17:53:45 - INFO - allennlp.training.trainer -   loss          |     0.693  |       N/A\n",
      "02/07/2019 17:53:45 - INFO - allennlp.training.trainer -   cpu_memory_MB |   724.333  |       N/A\n",
      "02/07/2019 17:53:45 - INFO - allennlp.training.trainer -   Epoch duration: 00:00:04\n",
      "02/07/2019 17:53:45 - INFO - allennlp.training.trainer -   Estimated training time remaining: 0:00:04\n",
      "02/07/2019 17:53:45 - INFO - allennlp.training.trainer -   Epoch 1/1\n",
      "02/07/2019 17:53:45 - INFO - allennlp.training.trainer -   Peak CPU memory usage MB: 1109.905408\n",
      "02/07/2019 17:53:45 - INFO - allennlp.training.trainer -   Training\n",
      "loss: 0.6821 ||: 100%|██████████| 5/5 [00:05<00:00,  1.00s/it]\n",
      "02/07/2019 17:53:50 - INFO - allennlp.training.trainer -                     Training |  Validation\n",
      "02/07/2019 17:53:50 - INFO - allennlp.training.trainer -   loss          |     0.682  |       N/A\n",
      "02/07/2019 17:53:50 - INFO - allennlp.training.trainer -   cpu_memory_MB |  1109.905  |       N/A\n",
      "02/07/2019 17:53:50 - INFO - allennlp.training.trainer -   Epoch duration: 00:00:05\n"
     ]
    }
   ],
   "source": [
    "metrics = trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generating Predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "AllenNLP is slightly lacking in its ability to convert datasets to predictions (though it has extensive support for converting single examples to predictions). Therefore, we'll write our own Predictor class to handle this job for us.\n",
    "\n",
    "Thankfully, a lot of the tools we used eariler can easily be extended to prediction. Here's how."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "from allennlp.data.iterators import DataIterator\n",
    "from tqdm import tqdm\n",
    "from scipy.special import expit # the sigmoid function\n",
    "\n",
    "def tonp(tsr): return tsr.detach().cpu().numpy()\n",
    "\n",
    "class Predictor:\n",
    "    def __init__(self, model: Model, iterator: DataIterator,\n",
    "                 cuda_device: int=-1) -> None:\n",
    "        self.model = model\n",
    "        self.iterator = iterator\n",
    "        self.cuda_device = cuda_device\n",
    "        \n",
    "    def _extract_data(self, batch) -> np.ndarray:\n",
    "        out_dict = self.model(**batch)\n",
    "        return expit(tonp(out_dict[\"class_logits\"]))\n",
    "    \n",
    "    def predict(self, ds: Iterable[Instance]) -> np.ndarray:\n",
    "        pred_generator = self.iterator(ds, num_epochs=1, shuffle=False)\n",
    "        self.model.eval()\n",
    "        pred_generator_tqdm = tqdm(pred_generator,\n",
    "                                   total=self.iterator.get_num_batches(ds))\n",
    "        preds = []\n",
    "        with torch.no_grad():\n",
    "            for batch in pred_generator_tqdm:\n",
    "                batch = nn_util.move_to_device(batch, self.cuda_device)\n",
    "                preds.append(self._extract_data(batch))\n",
    "        return np.concatenate(preds, axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll need a iterator that goes sequentially over our data. We'll use the "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "from allennlp.data.iterators import BasicIterator\n",
    "# iterate over the dataset without changing its order\n",
    "seq_iterator = BasicIterator(batch_size=64)\n",
    "seq_iterator.index_with(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5/5 [00:00<00:00, 18.65it/s]\n",
      "100%|██████████| 4/4 [00:00<00:00, 14.44it/s]\n"
     ]
    }
   ],
   "source": [
    "predictor = Predictor(model, seq_iterator, cuda_device=0 if USE_GPU else -1)\n",
    "train_preds = predictor.predict(train_ds) \n",
    "test_preds = predictor.predict(test_ds) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# A Final Note on Predictors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "AllenNLP also provides predictors that take strings as input and outputs model predictions. They're handy if you want to create simple demo or need to make predictions on entirely new data, but since we've already read data as datasets and want to preserve their order, we didn't use them above. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "from allennlp.predictors.sentence_tagger import SentenceTaggerPredictor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "tagger = SentenceTaggerPredictor(model, reader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'class_logits': [-0.07560957223176956,\n",
       "  0.03300512582063675,\n",
       "  -0.06693253666162491,\n",
       "  -0.001402825117111206,\n",
       "  -0.029230568557977676,\n",
       "  -0.06722232699394226],\n",
       " 'loss': 0.6762115359306335}"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tagger.predict(\"this tutorial was great!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'class_logits': [-0.07560957223176956,\n",
       "  0.03300512582063675,\n",
       "  -0.06693253666162491,\n",
       "  -0.001402825117111206,\n",
       "  -0.029230568557977676,\n",
       "  -0.06722232699394226],\n",
       " 'loss': 0.6762115359306335}"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tagger.predict(\"this tutorial was horrible!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Tags",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
